import requests
import csv
from bs4 import BeautifulSoup
import time

# URL
urls = []

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
    "Cache-Control": "no-cache",
    "DNT": "1",
    "Host": "www.zhongyoo.com",
    "Pragma": "no-cache",
    "Proxy-Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0",
}


def read_urls_from_file(file_path):
    with open(file_path, "r", encoding="utf-8") as file:
        for line in file:
            urls.append(line.strip())  # 去除每行末尾的换行符


# 从文件读取 URL
read_urls_from_file("urlsFather.txt")


def printToFile(data: str):
    with open("TCM-ID-output.html", "w", encoding="utf-8") as f:
        print(data, file=f)
    print("输出已经写入到文件", data)


def getFromUrl(url, index):
    if url == "":
        print("空url")
        return
    print("正在获取：" + url + "(" + str(index) + "/" + str(len(urls)) + ")")
    # 发送 GET 请求
    response = requests.get(url=url, headers=headers)
    soup = BeautifulSoup(response.text, "lxml")
    printToFile(soup.tbody)


# 调用函数
getFromUrl("https://www.bidd.group/TCMID/results.php?browse_formula_all=yes", 0)
